{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Advanced MLP - 2\n",
    "- Putting it altogether"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.datasets import mnist\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Activation, Dense, BatchNormalization, Dropout\n",
    "from keras import optimizers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# reshaping X data: (n, 28, 28) => (n, 784)\n",
    "X_train = X_train.reshape((X_train.shape[0], X_train.shape[1] * X_train.shape[2]))\n",
    "X_test = X_test.reshape((X_test.shape[0], X_test.shape[1] * X_test.shape[2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 784)\n",
      "(10000, 784)\n",
      "(60000,)\n",
      "(10000,)\n"
     ]
    }
   ],
   "source": [
    "# We use all training data and validate on all test data\n",
    "print(X_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training & Validating Model\n",
    "- Measures to improve training is applied simultaneously\n",
    "    - More training set\n",
    "    - Weight Initialization scheme\n",
    "    - Nonlinearity (Activation function)\n",
    "    - Optimizers: adaptvie\n",
    "    - Batch Normalization\n",
    "    - Dropout (Regularization)\n",
    "    - Model Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def mlp_model():\n",
    "    model = Sequential()\n",
    "    \n",
    "    model.add(Dense(50, input_shape = (784, ), kernel_initializer='he_normal'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Dense(50, kernel_initializer='he_normal'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Activation('relu'))    \n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Dense(50, kernel_initializer='he_normal'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Dense(50, kernel_initializer='he_normal'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Dense(10, kernel_initializer='he_normal'))\n",
    "    model.add(Activation('softmax'))\n",
    "    \n",
    "    adam = optimizers.Adam(lr = 0.001)\n",
    "    model.compile(optimizer = adam, loss = 'categorical_crossentropy', metrics = ['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create 5 models to ensemble\n",
    "model1 = KerasClassifier(build_fn = mlp_model, epochs = 100)\n",
    "model2 = KerasClassifier(build_fn = mlp_model, epochs = 100)\n",
    "model3 = KerasClassifier(build_fn = mlp_model, epochs = 100)\n",
    "model4 = KerasClassifier(build_fn = mlp_model, epochs = 100)\n",
    "model5 = KerasClassifier(build_fn = mlp_model, epochs = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ensemble_clf = VotingClassifier(estimators = [('model1', model1), ('model2', model2), ('model3', model3), ('model4', model4), ('model5', model5)], voting = 'soft')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ensemble_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9920/10000 [============================>.] - ETA: 0s"
     ]
    }
   ],
   "source": [
    "y_pred = ensemble_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc:  0.9801\n"
     ]
    }
   ],
   "source": [
    "print('Acc: ', accuracy_score(y_pred, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Final model produces over **98% of accuracy**, which is rather improved from before"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
